{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "ca898bb0-a0a3-4837-9508-bde5e9a3e4eb",
   "metadata": {},
   "outputs": [],
   "source": [
    "# !pip install polars\n",
    "# !pip install optuna\n",
    "# !pip install imblearn\n",
    "# !pip install lightgbm\n",
    "# !pip install catboost\n",
    "# !pip install xgboost\n",
    "# !pip install eli5\n",
    "# !pip install scikit-learn==1.2.0"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "76253482-b6c5-45fa-83bd-a882ac36904c",
   "metadata": {},
   "source": [
    "### Notes\n",
    "\n",
    "- base code is taken from [here](https://www.kaggle.com/code/greysky/isic-2024-only-tabular-data)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 29,
   "id": "c6fcb381-6ec3-42c7-a369-bb83fd5c23e2",
   "metadata": {},
   "outputs": [],
   "source": [
    "import os\n",
    "from glob import glob\n",
    "from pathlib import Path\n",
    "\n",
    "import numpy as np\n",
    "import pandas as pd\n",
    "import polars as pl\n",
    "import plotly.graph_objects as go\n",
    "\n",
    "from typing import List, Tuple, Dict, Optional\n",
    "from pydantic import BaseModel\n",
    "\n",
    "from tqdm import tqdm\n",
    "\n",
    "from sklearn.neighbors import LocalOutlierFactor\n",
    "from sklearn.model_selection import cross_val_score, StratifiedGroupKFold\n",
    "from sklearn.preprocessing import OneHotEncoder, LabelEncoder, StandardScaler\n",
    "from sklearn.metrics import roc_auc_score, silhouette_score\n",
    "from sklearn.ensemble import VotingClassifier\n",
    "from sklearn.cluster import KMeans\n",
    "from sklearn.inspection import permutation_importance\n",
    "\n",
    "from imblearn.under_sampling import RandomUnderSampler\n",
    "from imblearn.over_sampling import RandomOverSampler\n",
    "from imblearn.pipeline import Pipeline\n",
    "\n",
    "from catboost import CatBoostClassifier, Pool\n",
    "\n",
    "from matplotlib import pyplot as plt\n",
    "\n",
    "import lightgbm as lgb\n",
    "import catboost as cb\n",
    "import xgboost as xgb\n",
    "\n",
    "from PIL import Image\n",
    "\n",
    "from scipy.stats import ttest_ind, ttest_1samp, ttest_rel\n",
    "\n",
    "from eli5.permutation_importance import get_score_importances\n",
    "\n",
    "import optuna\n",
    "from optuna.trial import TrialState\n",
    "\n",
    "import warnings\n",
    "warnings.filterwarnings('ignore')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "id": "dbf9f926-a00a-4979-a62a-1e98839cc5b8",
   "metadata": {},
   "outputs": [],
   "source": [
    "root = Path('../data/original')\n",
    "\n",
    "train_path = root / 'train-metadata.csv'\n",
    "test_path = root / 'test-metadata.csv'\n",
    "subm_path = root / 'sample_submission.csv'\n",
    "\n",
    "id_col = 'isic_id'\n",
    "target_col = 'target'\n",
    "group_col = 'patient_id'\n",
    "\n",
    "err = 1e-5\n",
    "sampling_ratio = 0.01\n",
    "seed = 42\n",
    "\n",
    "\n",
    "num_cols = [\n",
    "    'age_approx',                        # Approximate age of patient at time of imaging.\n",
    "    'clin_size_long_diam_mm',            # Maximum diameter of the lesion (mm).+\n",
    "    'tbp_lv_A',                          # A inside  lesion.+\n",
    "    'tbp_lv_Aext',                       # A outside lesion.+\n",
    "    'tbp_lv_B',                          # B inside  lesion.+\n",
    "    'tbp_lv_Bext',                       # B outside lesion.+ \n",
    "    'tbp_lv_C',                          # Chroma inside  lesion.+\n",
    "    'tbp_lv_Cext',                       # Chroma outside lesion.+\n",
    "    'tbp_lv_H',                          # Hue inside the lesion; calculated as the angle of A* and B* in LAB* color space. Typical values range from 25 (red) to 75 (brown).+\n",
    "    'tbp_lv_Hext',                       # Hue outside lesion.+\n",
    "    'tbp_lv_L',                          # L inside lesion.+\n",
    "    'tbp_lv_Lext',                       # L outside lesion.+\n",
    "    'tbp_lv_areaMM2',                    # Area of lesion (mm^2).+\n",
    "    'tbp_lv_area_perim_ratio',           # Border jaggedness, the ratio between lesions perimeter and area. Circular lesions will have low values; irregular shaped lesions will have higher values. Values range 0-10.+\n",
    "    'tbp_lv_color_std_mean',             # Color irregularity, calculated as the variance of colors within the lesion's boundary.\n",
    "    'tbp_lv_deltaA',                     # Average A contrast (inside vs. outside lesion).+\n",
    "    'tbp_lv_deltaB',                     # Average B contrast (inside vs. outside lesion).+\n",
    "    'tbp_lv_deltaL',                     # Average L contrast (inside vs. outside lesion).+\n",
    "    'tbp_lv_deltaLB',                    #\n",
    "    'tbp_lv_deltaLBnorm',                # Contrast between the lesion and its immediate surrounding skin. Low contrast lesions tend to be faintly visible such as freckles; high contrast lesions tend to be those with darker pigment. Calculated as the average delta LB of the lesion relative to its immediate background in LAB* color space. Typical values range from 5.5 to 25.+\n",
    "    'tbp_lv_eccentricity',               # Eccentricity.+\n",
    "    'tbp_lv_minorAxisMM',                # Smallest lesion diameter (mm).+\n",
    "    'tbp_lv_nevi_confidence',            # Nevus confidence score (0-100 scale) is a convolutional neural network classifier estimated probability that the lesion is a nevus. The neural network was trained on approximately 57,000 lesions that were classified and labeled by a dermatologist.+,++\n",
    "    'tbp_lv_norm_border',                # Border irregularity (0-10 scale); the normalized average of border jaggedness and asymmetry.+\n",
    "    'tbp_lv_norm_color',                 # Color variation (0-10 scale); the normalized average of color asymmetry and color irregularity.+\n",
    "    'tbp_lv_perimeterMM',                # Perimeter of lesion (mm).+\n",
    "    'tbp_lv_radial_color_std_max',       # Color asymmetry, a measure of asymmetry of the spatial distribution of color within the lesion. This score is calculated by looking at the average standard deviation in LAB* color space within concentric rings originating from the lesion center. Values range 0-10.+\n",
    "    'tbp_lv_stdL',                       # Standard deviation of L inside  lesion.+\n",
    "    'tbp_lv_stdLExt',                    # Standard deviation of L outside lesion.+\n",
    "    'tbp_lv_symm_2axis',                 # Border asymmetry; a measure of asymmetry of the lesion's contour about an axis perpendicular to the lesion's most symmetric axis. Lesions with two axes of symmetry will therefore have low scores (more symmetric), while lesions with only one or zero axes of symmetry will have higher scores (less symmetric). This score is calculated by comparing opposite halves of the lesion contour over many degrees of rotation. The angle where the halves are most similar identifies the principal axis of symmetry, while the second axis of symmetry is perpendicular to the principal axis. Border asymmetry is reported as the asymmetry value about this second axis. Values range 0-10.+\n",
    "    'tbp_lv_symm_2axis_angle',           # Lesion border asymmetry angle.+\n",
    "    'tbp_lv_x',                          # X-coordinate of the lesion on 3D TBP.+\n",
    "    'tbp_lv_y',                          # Y-coordinate of the lesion on 3D TBP.+\n",
    "    'tbp_lv_z',                          # Z-coordinate of the lesion on 3D TBP.+\n",
    "]\n",
    "\n",
    "new_num_cols = [\n",
    "    'lesion_size_ratio',             # tbp_lv_minorAxisMM      / clin_size_long_diam_mm\n",
    "    'lesion_shape_index',            # tbp_lv_areaMM2          / tbp_lv_perimeterMM **2\n",
    "    'hue_contrast',                  # tbp_lv_H                - tbp_lv_Hext              abs\n",
    "    'luminance_contrast',            # tbp_lv_L                - tbp_lv_Lext              abs\n",
    "    'lesion_color_difference',       # tbp_lv_deltaA **2       + tbp_lv_deltaB **2 + tbp_lv_deltaL **2  sqrt  \n",
    "    'border_complexity',             # tbp_lv_norm_border      + tbp_lv_symm_2axis\n",
    "    'color_uniformity',              # tbp_lv_color_std_mean   / tbp_lv_radial_color_std_max\n",
    "\n",
    "    'position_distance_3d',          # tbp_lv_x **2 + tbp_lv_y **2 + tbp_lv_z **2  sqrt\n",
    "    'perimeter_to_area_ratio',       # tbp_lv_perimeterMM      / tbp_lv_areaMM2\n",
    "    'area_to_perimeter_ratio',       # tbp_lv_areaMM2          / tbp_lv_perimeterMM\n",
    "    'lesion_visibility_score',       # tbp_lv_deltaLBnorm      + tbp_lv_norm_color\n",
    "    'symmetry_border_consistency',   # tbp_lv_symm_2axis       * tbp_lv_norm_border\n",
    "    'consistency_symmetry_border',   # tbp_lv_symm_2axis       * tbp_lv_norm_border / (tbp_lv_symm_2axis + tbp_lv_norm_border)\n",
    "\n",
    "    'color_consistency',             # tbp_lv_stdL             / tbp_lv_Lext\n",
    "    'consistency_color',             # tbp_lv_stdL*tbp_lv_Lext / tbp_lv_stdL + tbp_lv_Lext\n",
    "    'size_age_interaction',          # clin_size_long_diam_mm  * age_approx\n",
    "    'hue_color_std_interaction',     # tbp_lv_H                * tbp_lv_color_std_mean\n",
    "    'lesion_severity_index',         # tbp_lv_norm_border      + tbp_lv_norm_color + tbp_lv_eccentricity / 3\n",
    "    'shape_complexity_index',        # border_complexity       + lesion_shape_index\n",
    "    'color_contrast_index',          # tbp_lv_deltaA + tbp_lv_deltaB + tbp_lv_deltaL + tbp_lv_deltaLBnorm\n",
    "\n",
    "    'log_lesion_area',               # tbp_lv_areaMM2          + 1  np.log\n",
    "    'normalized_lesion_size',        # clin_size_long_diam_mm  / age_approx\n",
    "    'mean_hue_difference',           # tbp_lv_H                + tbp_lv_Hext    / 2\n",
    "    'std_dev_contrast',              # tbp_lv_deltaA **2 + tbp_lv_deltaB **2 + tbp_lv_deltaL **2   / 3  np.sqrt\n",
    "    'color_shape_composite_index',   # tbp_lv_color_std_mean   + bp_lv_area_perim_ratio + tbp_lv_symm_2axis   / 3\n",
    "    'lesion_orientation_3d',         # tbp_lv_y                , tbp_lv_x  np.arctan2\n",
    "    'overall_color_difference',      # tbp_lv_deltaA           + tbp_lv_deltaB + tbp_lv_deltaL   / 3\n",
    "\n",
    "    'symmetry_perimeter_interaction',# tbp_lv_symm_2axis       * tbp_lv_perimeterMM\n",
    "    'comprehensive_lesion_index',    # tbp_lv_area_perim_ratio + tbp_lv_eccentricity + bp_lv_norm_color + tbp_lv_symm_2axis   / 4\n",
    "    'color_variance_ratio',          # tbp_lv_color_std_mean   / tbp_lv_stdLExt\n",
    "    'border_color_interaction',      # tbp_lv_norm_border      * tbp_lv_norm_color\n",
    "    'border_color_interaction_2',\n",
    "    'size_color_contrast_ratio',     # clin_size_long_diam_mm  / tbp_lv_deltaLBnorm\n",
    "    'age_normalized_nevi_confidence',# tbp_lv_nevi_confidence  / age_approx\n",
    "    'age_normalized_nevi_confidence_2',\n",
    "    'color_asymmetry_index',         # tbp_lv_symm_2axis       * tbp_lv_radial_color_std_max\n",
    "\n",
    "    'volume_approximation_3d',       # tbp_lv_areaMM2          * sqrt(tbp_lv_x**2 + tbp_lv_y**2 + tbp_lv_z**2)\n",
    "    'color_range',                   # abs(tbp_lv_L - tbp_lv_Lext) + abs(tbp_lv_A - tbp_lv_Aext) + abs(tbp_lv_B - tbp_lv_Bext)\n",
    "    'shape_color_consistency',       # tbp_lv_eccentricity     * tbp_lv_color_std_mean\n",
    "    'border_length_ratio',           # tbp_lv_perimeterMM      / pi * sqrt(tbp_lv_areaMM2 / pi)\n",
    "    'age_size_symmetry_index',       # age_approx              * clin_size_long_diam_mm * tbp_lv_symm_2axis\n",
    "    'index_age_size_symmetry',       # age_approx              * tbp_lv_areaMM2 * tbp_lv_symm_2axis\n",
    "]\n",
    "\n",
    "cat_cols = ['sex', 'anatom_site_general', 'tbp_tile_type', 'tbp_lv_location', 'tbp_lv_location_simple', 'attribution']\n",
    "norm_cols = [f'{col}_patient_norm' for col in num_cols + new_num_cols]\n",
    "\n",
    "# age_sex_norm_columns_raw = [\n",
    "#         'tbp_lv_H', 'age_approx','hue_contrast',\n",
    "#        'age_normalized_nevi_confidence_2', 'tbp_lv_deltaB',\n",
    "       # 'color_uniformity', 'tbp_lv_z',\n",
    "       # # 'clin_size_long_diam_mm', 'tbp_lv_y',\n",
    "       # # 'position_distance_3d', 'hue_contrast',\n",
    "       # # 'tbp_lv_stdLExt', 'mean_hue_difference',\n",
    "       # # 'age_normalized_nevi_confidence',\n",
    "       # # 'lesion_visibility_score',\n",
    "       # # 'position_distance_3d',\n",
    "       # 'tbp_lv_minorAxisMM', 'tbp_lv_Hext']\n",
    "# age_sex_norm_columns = [f'{col}_age_sex_norm' for col in age_sex_norm_columns_raw]\n",
    "\n",
    "\n",
    "# attribution_norm_cols = [f'{col}_attribution_norm' for col in attribution_norm_cols_row]\n",
    "special_cols = ['count_per_patient', \"tbp_lv_areaMM2_patient\", \"tbp_lv_areaMM2_bp\"] #'count_per_patient_bp'\n",
    "feature_cols = num_cols + new_num_cols + cat_cols + norm_cols + special_cols \n",
    "# + age_sex_norm_columns\n",
    "# + attribution_norm_cols\n",
    "\n",
    "# extra_features = ['0_channel_mean_patient', '1_channel_mean_patient', '2_channel_mean_patient', '0_channel_r', '1_channel_r', '2_channel_r']\n",
    "# feature_cols += extra_features"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "id": "b1b97052-78cc-4737-90cc-9e3e6c2bd9bb",
   "metadata": {},
   "outputs": [],
   "source": [
    "def set_seed(random_seed):\n",
    "    random.seed(random_seed)\n",
    "    torch.manual_seed(random_seed)\n",
    "    np.random.seed(random_seed)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "id": "b36f7ece-9b8f-4424-a8c9-81d38be2d24e",
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/tmp/ipykernel_3471571/2662678883.py:1: DtypeWarning: Columns (51,52) have mixed types. Specify dtype option on import or set low_memory=False.\n",
      "  train_df_raw = pd.read_csv(train_path)\n"
     ]
    }
   ],
   "source": [
    "train_df_raw = pd.read_csv(train_path)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "id": "9d87355c-cedd-4fae-a772-65bbf812b6af",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>isic_id</th>\n",
       "      <th>target</th>\n",
       "      <th>patient_id</th>\n",
       "      <th>age_approx</th>\n",
       "      <th>sex</th>\n",
       "      <th>anatom_site_general</th>\n",
       "      <th>clin_size_long_diam_mm</th>\n",
       "      <th>image_type</th>\n",
       "      <th>tbp_tile_type</th>\n",
       "      <th>tbp_lv_A</th>\n",
       "      <th>...</th>\n",
       "      <th>lesion_id</th>\n",
       "      <th>iddx_full</th>\n",
       "      <th>iddx_1</th>\n",
       "      <th>iddx_2</th>\n",
       "      <th>iddx_3</th>\n",
       "      <th>iddx_4</th>\n",
       "      <th>iddx_5</th>\n",
       "      <th>mel_mitotic_index</th>\n",
       "      <th>mel_thick_mm</th>\n",
       "      <th>tbp_lv_dnn_lesion_confidence</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>ISIC_0015670</td>\n",
       "      <td>0</td>\n",
       "      <td>IP_1235828</td>\n",
       "      <td>60.0</td>\n",
       "      <td>male</td>\n",
       "      <td>lower extremity</td>\n",
       "      <td>3.04</td>\n",
       "      <td>TBP tile: close-up</td>\n",
       "      <td>3D: white</td>\n",
       "      <td>20.244422</td>\n",
       "      <td>...</td>\n",
       "      <td>NaN</td>\n",
       "      <td>Benign</td>\n",
       "      <td>Benign</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>97.517282</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>ISIC_0015845</td>\n",
       "      <td>0</td>\n",
       "      <td>IP_8170065</td>\n",
       "      <td>60.0</td>\n",
       "      <td>male</td>\n",
       "      <td>head/neck</td>\n",
       "      <td>1.10</td>\n",
       "      <td>TBP tile: close-up</td>\n",
       "      <td>3D: white</td>\n",
       "      <td>31.712570</td>\n",
       "      <td>...</td>\n",
       "      <td>IL_6727506</td>\n",
       "      <td>Benign</td>\n",
       "      <td>Benign</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>3.141455</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>2 rows × 55 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "        isic_id  target  patient_id  age_approx   sex anatom_site_general  \\\n",
       "0  ISIC_0015670       0  IP_1235828        60.0  male     lower extremity   \n",
       "1  ISIC_0015845       0  IP_8170065        60.0  male           head/neck   \n",
       "\n",
       "   clin_size_long_diam_mm          image_type tbp_tile_type   tbp_lv_A  ...  \\\n",
       "0                    3.04  TBP tile: close-up     3D: white  20.244422  ...   \n",
       "1                    1.10  TBP tile: close-up     3D: white  31.712570  ...   \n",
       "\n",
       "    lesion_id  iddx_full  iddx_1  iddx_2  iddx_3  iddx_4  iddx_5  \\\n",
       "0         NaN     Benign  Benign     NaN     NaN     NaN     NaN   \n",
       "1  IL_6727506     Benign  Benign     NaN     NaN     NaN     NaN   \n",
       "\n",
       "   mel_mitotic_index  mel_thick_mm  tbp_lv_dnn_lesion_confidence  \n",
       "0                NaN           NaN                     97.517282  \n",
       "1                NaN           NaN                      3.141455  \n",
       "\n",
       "[2 rows x 55 columns]"
      ]
     },
     "execution_count": 5,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "train_df_raw.head(2)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "id": "5542efac-b1d7-40f2-a91e-53f58093cd4e",
   "metadata": {},
   "outputs": [],
   "source": [
    "def read_data(path):\n",
    "    return (\n",
    "        pl.read_csv(path)\n",
    "        .with_columns(\n",
    "            pl.col('age_approx').cast(pl.String).replace('NA', np.nan).cast(pl.Float64),\n",
    "        )\n",
    "        .with_columns(\n",
    "            pl.col(pl.Float64).fill_nan(pl.col(pl.Float64).median()), # You may want to impute test data with train\n",
    "        )\n",
    "        .with_columns(\n",
    "            lesion_size_ratio              = pl.col('tbp_lv_minorAxisMM') / pl.col('clin_size_long_diam_mm'),\n",
    "            lesion_shape_index             = pl.col('tbp_lv_areaMM2') / (pl.col('tbp_lv_perimeterMM') ** 2),\n",
    "            hue_contrast                   = (pl.col('tbp_lv_H') - pl.col('tbp_lv_Hext')).abs(),\n",
    "            luminance_contrast             = (pl.col('tbp_lv_L') - pl.col('tbp_lv_Lext')).abs(),\n",
    "            lesion_color_difference        = (pl.col('tbp_lv_deltaA') ** 2 + pl.col('tbp_lv_deltaB') ** 2 + pl.col('tbp_lv_deltaL') ** 2).sqrt(),\n",
    "            border_complexity              = pl.col('tbp_lv_norm_border') + pl.col('tbp_lv_symm_2axis'),\n",
    "            color_uniformity               = pl.col('tbp_lv_color_std_mean') / (pl.col('tbp_lv_radial_color_std_max') + err),\n",
    "        )\n",
    "        .with_columns(\n",
    "            position_distance_3d           = (pl.col('tbp_lv_x') ** 2 + pl.col('tbp_lv_y') ** 2 + pl.col('tbp_lv_z') ** 2).sqrt(),\n",
    "            perimeter_to_area_ratio        = pl.col('tbp_lv_perimeterMM') / pl.col('tbp_lv_areaMM2'),\n",
    "            area_to_perimeter_ratio        = pl.col('tbp_lv_areaMM2') / pl.col('tbp_lv_perimeterMM'),\n",
    "            lesion_visibility_score        = pl.col('tbp_lv_deltaLBnorm') + pl.col('tbp_lv_norm_color'),\n",
    "            combined_anatomical_site       = pl.col('anatom_site_general') + '_' + pl.col('tbp_lv_location'),\n",
    "            symmetry_border_consistency    = pl.col('tbp_lv_symm_2axis') * pl.col('tbp_lv_norm_border'),\n",
    "            consistency_symmetry_border    = pl.col('tbp_lv_symm_2axis') * pl.col('tbp_lv_norm_border') / (pl.col('tbp_lv_symm_2axis') + pl.col('tbp_lv_norm_border')),\n",
    "        )\n",
    "        .with_columns(\n",
    "            color_consistency              = pl.col('tbp_lv_stdL') / pl.col('tbp_lv_Lext'),\n",
    "            consistency_color              = pl.col('tbp_lv_stdL') * pl.col('tbp_lv_Lext') / (pl.col('tbp_lv_stdL') + pl.col('tbp_lv_Lext')),\n",
    "            size_age_interaction           = pl.col('clin_size_long_diam_mm') * pl.col('age_approx'),\n",
    "            hue_color_std_interaction      = pl.col('tbp_lv_H') * pl.col('tbp_lv_color_std_mean'),\n",
    "            lesion_severity_index          = (pl.col('tbp_lv_norm_border') + pl.col('tbp_lv_norm_color') + pl.col('tbp_lv_eccentricity')) / 3,\n",
    "            shape_complexity_index         = pl.col('border_complexity') + pl.col('lesion_shape_index'),\n",
    "            color_contrast_index           = pl.col('tbp_lv_deltaA') + pl.col('tbp_lv_deltaB') + pl.col('tbp_lv_deltaL') + pl.col('tbp_lv_deltaLBnorm'),\n",
    "        )\n",
    "        .with_columns(\n",
    "            log_lesion_area                = (pl.col('tbp_lv_areaMM2') + 1).log(),\n",
    "            normalized_lesion_size         = pl.col('clin_size_long_diam_mm') / pl.col('age_approx'),\n",
    "            mean_hue_difference            = (pl.col('tbp_lv_H') + pl.col('tbp_lv_Hext')) / 2,\n",
    "            std_dev_contrast               = ((pl.col('tbp_lv_deltaA') ** 2 + pl.col('tbp_lv_deltaB') ** 2 + pl.col('tbp_lv_deltaL') ** 2) / 3).sqrt(),\n",
    "            color_shape_composite_index    = (pl.col('tbp_lv_color_std_mean') + pl.col('tbp_lv_area_perim_ratio') + pl.col('tbp_lv_symm_2axis')) / 3,\n",
    "            lesion_orientation_3d          = pl.arctan2(pl.col('tbp_lv_y'), pl.col('tbp_lv_x')),\n",
    "            overall_color_difference       = (pl.col('tbp_lv_deltaA') + pl.col('tbp_lv_deltaB') + pl.col('tbp_lv_deltaL')) / 3,\n",
    "        )\n",
    "        .with_columns(\n",
    "            symmetry_perimeter_interaction = pl.col('tbp_lv_symm_2axis') * pl.col('tbp_lv_perimeterMM'),\n",
    "            comprehensive_lesion_index     = (pl.col('tbp_lv_area_perim_ratio') + pl.col('tbp_lv_eccentricity') + pl.col('tbp_lv_norm_color') + pl.col('tbp_lv_symm_2axis')) / 4,\n",
    "            color_variance_ratio           = pl.col('tbp_lv_color_std_mean') / pl.col('tbp_lv_stdLExt'),\n",
    "            border_color_interaction       = pl.col('tbp_lv_norm_border') * pl.col('tbp_lv_norm_color'),\n",
    "            border_color_interaction_2     = pl.col('tbp_lv_norm_border') * pl.col('tbp_lv_norm_color') / (pl.col('tbp_lv_norm_border') + pl.col('tbp_lv_norm_color')),\n",
    "            size_color_contrast_ratio      = pl.col('clin_size_long_diam_mm') / pl.col('tbp_lv_deltaLBnorm'),\n",
    "            age_normalized_nevi_confidence = pl.col('tbp_lv_nevi_confidence') / pl.col('age_approx'),\n",
    "            age_normalized_nevi_confidence_2 = (pl.col('clin_size_long_diam_mm')**2 + pl.col('age_approx')**2).sqrt(),\n",
    "            color_asymmetry_index          = pl.col('tbp_lv_radial_color_std_max') * pl.col('tbp_lv_symm_2axis'),\n",
    "        )\n",
    "        .with_columns(\n",
    "            volume_approximation_3d        = pl.col('tbp_lv_areaMM2') * (pl.col('tbp_lv_x')**2 + pl.col('tbp_lv_y')**2 + pl.col('tbp_lv_z')**2).sqrt(),\n",
    "            color_range                    = (pl.col('tbp_lv_L') - pl.col('tbp_lv_Lext')).abs() + (pl.col('tbp_lv_A') - pl.col('tbp_lv_Aext')).abs() + (pl.col('tbp_lv_B') - pl.col('tbp_lv_Bext')).abs(),\n",
    "            shape_color_consistency        = pl.col('tbp_lv_eccentricity') * pl.col('tbp_lv_color_std_mean'),\n",
    "            border_length_ratio            = pl.col('tbp_lv_perimeterMM') / (2 * np.pi * (pl.col('tbp_lv_areaMM2') / np.pi).sqrt()),\n",
    "            age_size_symmetry_index        = pl.col('age_approx') * pl.col('clin_size_long_diam_mm') * pl.col('tbp_lv_symm_2axis'),\n",
    "            index_age_size_symmetry        = pl.col('age_approx') * pl.col('tbp_lv_areaMM2') * pl.col('tbp_lv_symm_2axis'),\n",
    "        )\n",
    "        .with_columns(\n",
    "            ((pl.col(col) - pl.col(col).mean().over('patient_id')) / (pl.col(col).std().over('patient_id') + err)).alias(f'{col}_patient_norm') for col in (num_cols + new_num_cols)\n",
    "        )\n",
    "        # .with_columns(\n",
    "        #     ((pl.col(col) - pl.col(col).mean().over(['attribution'])) / (pl.col(col).std().over(['attribution']) + err)).alias(f'{col}_attribution_norm') for col in attribution_norm_cols_row\n",
    "        # )\n",
    "        .with_columns(\n",
    "            count_per_patient = pl.col('isic_id').count().over('patient_id'),\n",
    "        )\n",
    "\n",
    "        .with_columns(\n",
    "            tbp_lv_areaMM2_patient = pl.col('tbp_lv_areaMM2').sum().over('patient_id'),\n",
    "        )\n",
    "        .with_columns(\n",
    "            tbp_lv_areaMM2_bp = pl.col('tbp_lv_areaMM2').sum().over(['patient_id', 'anatom_site_general']),\n",
    "        )\n",
    "\n",
    "\n",
    "        # .with_columns(\n",
    "        #     tbp_lv_minorAxisMM_age_norm = pl.col('tbp_lv_minorAxisMM') / pl.col('tbp_lv_minorAxisMM').mean().over(['age_approx', 'sex']),\n",
    "        # )\n",
    "        # .with_columns(\n",
    "        #     ((pl.col(col) / pl.col(col).mean().over(['age_approx', 'sex']))).alias(f'{col}_age_sex_norm') for col in age_sex_norm_columns_raw\n",
    "        # )\n",
    "\n",
    "        .with_columns(\n",
    "            pl.col(cat_cols).cast(pl.Categorical),\n",
    "        )\n",
    "        .to_pandas()\n",
    "        .set_index(id_col)\n",
    "    )"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "id": "3ef80da6-3f31-4ed6-a461-6869587ebf86",
   "metadata": {},
   "outputs": [],
   "source": [
    "def preprocess(df_train, df_test):\n",
    "    global cat_cols\n",
    "    \n",
    "    encoder = OneHotEncoder(sparse_output=False, dtype=np.int32, handle_unknown='ignore')\n",
    "    encoder.fit(df_train[cat_cols])\n",
    "    \n",
    "    new_cat_cols = [f'onehot_{i}' for i in range(len(encoder.get_feature_names_out()))]\n",
    "\n",
    "    df_train[new_cat_cols] = encoder.transform(df_train[cat_cols])\n",
    "    df_train[new_cat_cols] = df_train[new_cat_cols].astype('category')\n",
    "\n",
    "    df_test[new_cat_cols] = encoder.transform(df_test[cat_cols])\n",
    "    df_test[new_cat_cols] = df_test[new_cat_cols].astype('category')\n",
    "\n",
    "    for col in cat_cols:\n",
    "        feature_cols.remove(col)\n",
    "\n",
    "    feature_cols.extend(new_cat_cols)\n",
    "    cat_cols = new_cat_cols\n",
    "    \n",
    "    return df_train, df_test"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "id": "da70c541-5efd-47c0-9749-e0a71f4e6db7",
   "metadata": {},
   "outputs": [],
   "source": [
    "def custom_metric_raw(y_hat, y_true):\n",
    "    min_tpr = 0.80\n",
    "    max_fpr = abs(1 - min_tpr)\n",
    "    \n",
    "    v_gt = abs(y_true - 1)\n",
    "    v_pred = np.array([1.0 - x for x in y_hat])\n",
    "    \n",
    "    partial_auc_scaled = roc_auc_score(v_gt, v_pred, max_fpr=max_fpr)\n",
    "    partial_auc = 0.5 * max_fpr**2 + (max_fpr - 0.5 * max_fpr**2) / (1.0 - 0.5) * (partial_auc_scaled - 0.5)\n",
    "    \n",
    "    return partial_auc\n",
    "\n",
    "\n",
    "def custom_metric(estimator, X, y_true):\n",
    "    y_hat = estimator.predict_proba(X)[:, 1]\n",
    "    partial_auc = custom_metric_raw(y_hat, y_true)\n",
    "    return partial_auc\n",
    "    "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "id": "c5fab7c3-4b4a-490b-b7f8-c014b10e694f",
   "metadata": {},
   "outputs": [],
   "source": [
    "def get_lof_score(df_train, top_lof_features, output_col_name=\"of\", lof_column='patient_id'):\n",
    "    scalled_array = StandardScaler().fit_transform(df_train[top_lof_features])\n",
    "    outiler_factors = []\n",
    "    for tmp_col_v in tqdm(df_train[lof_column].unique()):\n",
    "        mask = (df_train[lof_column] == tmp_col_v).values\n",
    "        if sum(mask) < 3:\n",
    "            continue\n",
    "        patient_emb = scalled_array[mask]\n",
    "        clf = LocalOutlierFactor(n_neighbors=min(sum(mask), 30))\n",
    "        clf.fit_predict(patient_emb)\n",
    "        \n",
    "    \n",
    "        outiler_factors.append(pd.DataFrame(\n",
    "            {\"isic_id\": df_train[mask].index.values,\n",
    "            output_col_name: clf.negative_outlier_factor_}))\n",
    "    \n",
    "    outiler_factors = pd.concat(outiler_factors).reset_index(drop=True)\n",
    "    \n",
    "    df_train = df_train.merge(outiler_factors.set_index('isic_id'), how=\"left\", left_index=True, right_index=True)\n",
    "    df_train[output_col_name] = df_train[output_col_name].fillna(-1)\n",
    "    return df_train"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "id": "39751d05-302e-4479-8b08-b667306dfc10",
   "metadata": {},
   "outputs": [],
   "source": [
    "top_lof_features = [\n",
    "        'tbp_lv_H','hue_contrast',\n",
    "       'age_normalized_nevi_confidence_2', 'tbp_lv_deltaB',\n",
    "       'color_uniformity', 'tbp_lv_z',\n",
    "       'clin_size_long_diam_mm', 'tbp_lv_y',\n",
    "       'position_distance_3d', 'hue_contrast',\n",
    "       'tbp_lv_stdLExt', 'mean_hue_difference',\n",
    "       'age_normalized_nevi_confidence',\n",
    "       'lesion_visibility_score',\n",
    "       'position_distance_3d',\n",
    "       'tbp_lv_minorAxisMM', 'tbp_lv_Hext']"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "id": "93df8e0f-9754-4a50-a54d-dd2839913982",
   "metadata": {},
   "outputs": [],
   "source": [
    "df_train = read_data(train_path)\n",
    "df_test = read_data(test_path)\n",
    "df_subm = pd.read_csv(subm_path, index_col=id_col)\n",
    "\n",
    "df_train, df_test = preprocess(df_train, df_test)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "id": "576ccc00-7508-46cb-94d5-310f7019d41c",
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|██████████| 1042/1042 [05:07<00:00,  3.38it/s]\n"
     ]
    }
   ],
   "source": [
    "df_train = get_lof_score(df_train, top_lof_features, output_col_name=\"of\", lof_column='patient_id')\n",
    "feature_cols += ['of']"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "id": "c74ed673-8760-40e2-b4e3-463fc5496c5f",
   "metadata": {},
   "outputs": [],
   "source": [
    "### get predictions from model based on external data\n",
    "old_data_model_preds = pd.read_parquet('../data/artifacts/old_data_model_forecast_large.parquet') #old_data_model_forecast old_data_model_forecast\n",
    "df_train = df_train.merge(old_data_model_preds, how=\"left\", on=[\"isic_id\"])\n",
    "\n",
    "feature_cols += ['old_set_0', 'old_set_1', 'old_set_2']\n",
    "\n",
    "df_train = df_train.merge(\n",
    "    df_train.groupby(\"patient_id\").agg(**{\n",
    "        \"old_set_0_m\": pd.NamedAgg('old_set_0', 'mean'),\n",
    "        \"old_set_1_m\": pd.NamedAgg('old_set_1', 'mean'),\n",
    "        \"old_set_2_m\": pd.NamedAgg('old_set_2', 'mean')\n",
    "    }).reset_index(), how=\"left\", on=[\"patient_id\"])\n",
    "\n",
    "df_train['old_set_0_m'] = df_train['old_set_0'] / df_train['old_set_0_m']\n",
    "df_train['old_set_1_m'] = df_train['old_set_1'] / df_train['old_set_1_m']\n",
    "df_train['old_set_2_m'] = df_train['old_set_2'] / df_train['old_set_2_m']\n",
    "\n",
    "feature_cols += ['old_set_0_m', 'old_set_1_m', 'old_set_2_m']"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "id": "1aba2c81-ff4b-4884-ac6e-d85ccf221aab",
   "metadata": {},
   "outputs": [],
   "source": [
    "###get oof model predictions (eva)\n",
    "oof_forecasts = pd.read_parquet('../data/artifacts/oof_forecasts_eva_base.parquet')\n",
    "\n",
    "# tmp_predictions_all__pr\n",
    "df_train = df_train.merge(\n",
    "    oof_forecasts[['isic_id', 'tmp_predictions_all__pr']].rename(columns={\n",
    "        'tmp_predictions_all__pr': 'oof_eva_score'\n",
    "    }), how=\"left\", on=['isic_id'])\n",
    "\n",
    "\n",
    "df_train = df_train.merge(\n",
    "    df_train.groupby(\"patient_id\").agg(**{\n",
    "        \"oof_eva_score_m\": pd.NamedAgg('oof_eva_score', 'mean')\n",
    "    }).reset_index(), how=\"left\", on=[\"patient_id\"])\n",
    "\n",
    "df_train['oof_eva_score_m'] = df_train['oof_eva_score'] / df_train['oof_eva_score_m']\n",
    "\n",
    "feature_cols += ['oof_eva_score', 'oof_eva_score_m']"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 23,
   "id": "b6f02296-47fd-40d2-aae6-95c97ad64ed4",
   "metadata": {},
   "outputs": [],
   "source": [
    "###get oof model predictions (edgenext)\n",
    "oof_forecasts = pd.read_parquet('../data/artifacts/oof_forecasts_edgenext_base.parquet') # oof_forecasts_edgenext_base\n",
    "\n",
    "# tmp_predictions_all__pr\n",
    "df_train = df_train.merge(\n",
    "    oof_forecasts[['isic_id', 'tmp_predictions_all__pr']].rename(columns={\n",
    "        'tmp_predictions_all__pr': 'oof_edgenext_score'\n",
    "    }), how=\"left\", on=['isic_id'])\n",
    "\n",
    "df_train = df_train.merge(\n",
    "    df_train.groupby(\"patient_id\").agg(**{\n",
    "        \"oof_edgenext_score_m\": pd.NamedAgg('oof_edgenext_score', 'mean')\n",
    "    }).reset_index(), how=\"left\", on=[\"patient_id\"])\n",
    "\n",
    "df_train['oof_edgenext_score_m'] = df_train['oof_edgenext_score'] / df_train['oof_edgenext_score_m']\n",
    "\n",
    "feature_cols += ['oof_edgenext_score', 'oof_edgenext_score_m']"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 24,
   "id": "af8291ed-0c2b-4f52-9198-0177de4e4e6f",
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/tmp/ipykernel_3471571/3335836358.py:3: FutureWarning: The default of observed=False is deprecated and will be changed to True in a future version of pandas. Pass observed=False to retain current behavior or observed=True to adopt the future default and silence this warning.\n",
      "  df_train.groupby(['attribution', 'tbp_lv_location', 'age_approx']).agg(**{\n"
     ]
    }
   ],
   "source": [
    "###get oof based aggs\n",
    "df_train = df_train.merge(\n",
    "    df_train.groupby(['attribution', 'tbp_lv_location', 'age_approx']).agg(**{\n",
    "        \"oof_edgenext_score_mean_tmp\": pd.NamedAgg('oof_edgenext_score', 'mean'),\n",
    "        \"oof_edgenext_score_std_tmp\": pd.NamedAgg('oof_edgenext_score', 'std'),\n",
    "        \"oof_eva_score_mean_tmp\": pd.NamedAgg('oof_eva_score', 'mean'),\n",
    "        \"oof_eva_score_std_tmp\": pd.NamedAgg('oof_eva_score', 'std')\n",
    "    }).reset_index(), how=\"left\", on=['attribution', 'tbp_lv_location', 'age_approx'])\n",
    "\n",
    "df_train[\"oof_edgenext_score_mean_tmp\"] = (\n",
    "    df_train[\"oof_edgenext_score\"] - df_train[\"oof_edgenext_score_mean_tmp\"]) / df_train[\"oof_edgenext_score_std_tmp\"]\n",
    "df_train[\"oof_eva_score_mean_tmp\"] = (\n",
    "    df_train[\"oof_eva_score\"] - df_train[\"oof_eva_score_mean_tmp\"]) / df_train[\"oof_eva_score_std_tmp\"]\n",
    "\n",
    "feature_cols += ['oof_edgenext_score_mean_tmp', 'oof_eva_score_mean_tmp']"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 25,
   "id": "66bddcf0-fd6a-4f7a-8696-cc0e390d9a3c",
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/opt/conda/lib/python3.11/site-packages/sklearn/cluster/_kmeans.py:870: FutureWarning: The default value of `n_init` will change from 10 to 'auto' in 1.4. Set the value of `n_init` explicitly to suppress the warning\n",
      "  warnings.warn(\n"
     ]
    }
   ],
   "source": [
    "scaler = StandardScaler()\n",
    "X_scaled = scaler.fit_transform(df_train[top_lof_features])\n",
    "\n",
    "np.random.seed(42)\n",
    "kmeans = KMeans(n_clusters=29, random_state=42)\n",
    "cluster_labels = kmeans.fit_predict(X_scaled)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 26,
   "id": "302444cd-58c1-4b28-a886-bfc1edf8c5fa",
   "metadata": {},
   "outputs": [],
   "source": [
    "columns_for_cluster_culculations = [\n",
    "    \"tbp_lv_minorAxisMM\", \"tbp_lv_norm_border\", \"tbp_lv_norm_color\", \"tbp_lv_symm_2axis\",\n",
    "    \"tbp_lv_radial_color_std_max\", \"lesion_visibility_score\", \"mean_hue_difference\", \"clin_size_long_diam_mm\",\n",
    "    \"color_uniformity\", \"hue_contrast\", \"tbp_lv_H\", \"oof_eva_score\", \"oof_edgenext_score\",\n",
    "    'tbp_lv_Aext', 'tbp_lv_Bext', 'tbp_lv_Cext', 'tbp_lv_Lext', \n",
    "    'tbp_lv_deltaLBnorm', 'tbp_lv_stdLExt', 'position_distance_3d', 'color_contrast_index',\n",
    "    'age_normalized_nevi_confidence_2', 'volume_approximation_3d', 'age_approx_patient_norm', 'tbp_lv_H_patient_norm',\n",
    "    'tbp_lv_Hext_patient_norm', 'tbp_lv_L_patient_norm', 'tbp_lv_stdLExt_patient_norm', 'tbp_lv_symm_2axis_angle_patient_norm',\n",
    "    'tbp_lv_x_patient_norm', 'tbp_lv_y_patient_norm', 'tbp_lv_z_patient_norm', 'lesion_size_ratio_patient_norm', 'hue_contrast_patient_norm',\n",
    "    'color_uniformity_patient_norm', 'position_distance_3d_patient_norm', 'size_age_interaction_patient_norm', 'mean_hue_difference_patient_norm',\n",
    "    'std_dev_contrast_patient_norm', 'lesion_orientation_3d_patient_norm', \n",
    "    'volume_approximation_3d_patient_norm', 'color_range_patient_norm', 'tbp_lv_areaMM2_patient', 'tbp_lv_areaMM2_bp', 'of',\n",
    "    'old_set_0_m', 'old_set_1_m', 'oof_eva_score_m', 'oof_edgenext_score_m', 'oof_edgenext_score_mean_tmp', 'oof_eva_score_mean_tmp',\n",
    "]\n",
    "\n",
    "df_train['cluster_labels'] = cluster_labels\n",
    "agg_config = {\n",
    "    f\"{i}__mean\": pd.NamedAgg(i, \"mean\")\n",
    "        for i in columns_for_cluster_culculations\n",
    "}\n",
    "agg_config.update({\n",
    "    f\"{i}__std\": pd.NamedAgg(i, \"std\")\n",
    "        for i in columns_for_cluster_culculations\n",
    "})\n",
    "df_train = df_train.merge(\n",
    "    df_train.groupby(\"cluster_labels\").agg(**agg_config), how=\"left\", on=\"cluster_labels\")\n",
    "\n",
    "for col in columns_for_cluster_culculations:\n",
    "    df_train[f\"{col}__cluster\"] = (df_train[col] - df_train[f'{col}__mean']) / df_train[f'{col}__std']\n",
    "    feature_cols += [f\"{col}__cluster\"]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 27,
   "id": "ef6d95f2-d5f8-41d2-8982-1fd885b102e6",
   "metadata": {},
   "outputs": [],
   "source": [
    "def run_model_old(cb_params, reduce=True, n_rounds=10, columns_to_drop=None) -> float:\n",
    "    metric_list = []\n",
    "    columns_to_drop = [] if columns_to_drop is None else columns_to_drop\n",
    "    for random_seed in range(1, n_rounds):\n",
    "        tsp = StratifiedGroupKFold(5, shuffle=True, random_state=random_seed)\n",
    "        metrics_ev_df = []\n",
    "        test_forecast = []\n",
    "        val_forecast = []\n",
    "        for fold_n, (train_index, val_index) in enumerate(tsp.split(df_train, y=df_train.target, groups=df_train[group_col])):\n",
    "            train_slice_x = df_train.iloc[train_index][[i for i in feature_cols if i not in columns_to_drop]].reset_index(drop=True)\n",
    "            val_slice_x = df_train.iloc[val_index][[i for i in feature_cols if i not in columns_to_drop]].reset_index(drop=True)\n",
    "            \n",
    "            train_slice_y = df_train.iloc[train_index]['target'].reset_index(drop=True)\n",
    "            val_slice_y = df_train.iloc[val_index]['target'].reset_index(drop=True)\n",
    "        \n",
    "            cb_model = Pipeline([\n",
    "                ('sampler_1', RandomOverSampler(sampling_strategy= 0.003 , random_state=seed)),\n",
    "                ('sampler_2', RandomUnderSampler(sampling_strategy=sampling_ratio , random_state=seed)),\n",
    "                ('classifier', cb.CatBoostClassifier(**cb_params)),\n",
    "            ])\n",
    "            \n",
    "            cb_model.fit(train_slice_x, train_slice_y)\n",
    "            preds_cb = cb_model.predict_proba(val_slice_x)[:, 1]\n",
    "            metric = custom_metric_raw(preds_cb, val_slice_y.values)\n",
    "            metric_list.append(metric)\n",
    "\n",
    "    if reduce:\n",
    "        return np.mean(metric_list)\n",
    "    else:\n",
    "        return metric_list"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 28,
   "id": "e4a71423-d15c-45ee-bcf0-40e43863389e",
   "metadata": {},
   "outputs": [],
   "source": [
    "class ModelConfigCB(BaseModel):\n",
    "    iterations: int = 1000\n",
    "    learning_rate: float = 0.06936242010150652\n",
    "    l2_leaf_reg: float = 6.216113851699493\n",
    "    loss_function: str = \"Logloss\"\n",
    "    bagging_temperature: float = 1\n",
    "    random_seed: int = seed\n",
    "    border_count: int = 128\n",
    "    grow_policy: str = \"SymmetricTree\" #Depthwise Lossguide\n",
    "    min_data_in_leaf: int = 24\n",
    "    depth: int  = 7\n",
    "    do_sample: bool = True\n",
    "\n",
    "model_config_cb = ModelConfigCB()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 31,
   "id": "95e9afd9-30a7-4dbf-be36-c9c23fbba406",
   "metadata": {},
   "outputs": [],
   "source": [
    "def optimise_catboost(trial: optuna.Trial, test=False, reduce=True, columns_to_drop=None, n_rounds=10, cat_cols=None, get_fe=False) -> float:\n",
    "    cat_cols = [] if cat_cols is None else cat_cols\n",
    "    columns_to_drop = [] if columns_to_drop is None else columns_to_drop\n",
    "    model_config_cb = ModelConfigCB(\n",
    "        iterations = 2000,\n",
    "        learning_rate = trial.suggest_float('learning_rate', 0.01, 0.08) if not test else trial.get('learning_rate'),\n",
    "        l2_leaf_reg = trial.suggest_float('l2_leaf_reg', 1, 20) if not test else trial.get('l2_leaf_reg'),\n",
    "        random_strength = trial.suggest_float('random_strength', 0, 5) if not test else trial.get('random_strength'),\n",
    "        loss_function = \"Logloss\",\n",
    "        depth = trial.suggest_int('depth', 2, 8) if not test else trial.get('depth'),\n",
    "        bagging_temperature = trial.suggest_float('bagging_temperature', 0, 10) if not test else trial.get('bagging_temperature'),\n",
    "        border_count = trial.suggest_categorical('border_count', [128, 256]) if not test else trial.get('border_count'),\n",
    "        grow_policy = trial.suggest_categorical('grow_policy', [\"SymmetricTree\", \"Depthwise\", \"Lossguide\"]) if not test else trial.get('grow_policy'),\n",
    "        random_seed=42,\n",
    "        min_data_in_leaf = trial.suggest_int('min_data_in_leaf', 8, 40) if not test else trial.get('min_data_in_leaf'),\n",
    "    )\n",
    "\n",
    "    metric_list = []\n",
    "    models = []\n",
    "    all_val_df_tmp = []\n",
    "    score_decreases_list = []\n",
    "    for random_seed in range(1, n_rounds):\n",
    "        tsp = StratifiedGroupKFold(5, shuffle=True, random_state=random_seed)\n",
    "        metrics_ev_df = []\n",
    "        test_forecast = []\n",
    "        val_forecast = []\n",
    "        for fold_n, (train_index, val_index) in enumerate(tsp.split(df_train, y=df_train.target, groups=df_train[group_col])):\n",
    "            train_slice_x = df_train.iloc[train_index][[i for i in feature_cols if i not in columns_to_drop]].reset_index(drop=True)\n",
    "            val_slice_x = df_train.iloc[val_index][[i for i in feature_cols if i not in columns_to_drop]].reset_index(drop=True)\n",
    "\n",
    "            train_slice_y = df_train.iloc[train_index]['target'].reset_index(drop=True)\n",
    "            val_slice_y = df_train.iloc[val_index]['target'].reset_index(drop=True)\n",
    "            \n",
    "            if model_config_cb.do_sample:\n",
    "                cb_model = Pipeline([\n",
    "                    ('sampler_1', RandomOverSampler(sampling_strategy= 0.003 , random_state=random_seed)),\n",
    "                    ('sampler_2', RandomUnderSampler(sampling_strategy=sampling_ratio , random_state=random_seed)),\n",
    "                ])\n",
    "                \n",
    "                train_slice_x, train_slice_y = cb_model.fit_resample(train_slice_x, train_slice_y)\n",
    "            \n",
    "        \n",
    "            clf_catboost = CatBoostClassifier(\n",
    "                loss_function=model_config_cb.loss_function,\n",
    "                eval_metric='AUC',\n",
    "                task_type='GPU',\n",
    "                learning_rate=model_config_cb.learning_rate,\n",
    "                od_wait=100,\n",
    "                random_state=random_seed,\n",
    "                depth=model_config_cb.depth,\n",
    "                l2_leaf_reg=model_config_cb.l2_leaf_reg,\n",
    "                min_data_in_leaf=model_config_cb.min_data_in_leaf,\n",
    "                bagging_temperature=model_config_cb.bagging_temperature,\n",
    "                border_count=model_config_cb.border_count,\n",
    "                grow_policy=model_config_cb.grow_policy, \n",
    "                devices='0',\n",
    "                iterations=model_config_cb.iterations,\n",
    "            )\n",
    "        \n",
    "            train_pool = Pool(train_slice_x, train_slice_y.values, cat_features=cat_cols) \n",
    "            val_pool = Pool(val_slice_x, val_slice_y.values, cat_features=cat_cols) \n",
    "        \n",
    "            clf_catboost.fit(train_pool, eval_set=val_pool,verbose=False)\n",
    "            preds_cb = clf_catboost.predict_proba(val_slice_x)[:, 1]\n",
    "            metric = custom_metric_raw(preds_cb, val_slice_y.values)\n",
    "            metric_list.append(metric)\n",
    "            models.append(clf_catboost)\n",
    "            \n",
    "            \n",
    "            \n",
    "            # def make_score(clf, columns_names):\n",
    "            #     def score(X, y):\n",
    "            #         predictions_tmp = clf.predict_proba(pd.DataFrame(X, columns=columns_names), thread_count=-1)[:, 1]\n",
    "            #         return custom_metric_raw(predictions_tmp, y)\n",
    "            #     return score\n",
    "\n",
    "            if get_fe:\n",
    "                score_decreases = permutation_importance(clf_catboost, val_slice_x, val_slice_y.values,\n",
    "                           n_repeats=4,\n",
    "                           random_state=0)\n",
    "\n",
    "                score_decreases_list.append(score_decreases)\n",
    "                \n",
    "            val_df_tmp = df_train.iloc[val_index].reset_index(drop=False)\n",
    "            val_df_tmp['predictions'] = preds_cb\n",
    "            val_df_tmp['random_seed'] = random_seed\n",
    "            all_val_df_tmp.append(val_df_tmp)\n",
    "\n",
    "    if reduce:\n",
    "        return np.mean(metric_list), models, pd.concat(all_val_df_tmp).reset_index(drop=True), score_decreases_list\n",
    "    else:\n",
    "        return metric_list, models, pd.concat(all_val_df_tmp).reset_index(drop=True), score_decreases_list"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 32,
   "id": "e8d8d367-96a2-4136-8ee4-e4287f420421",
   "metadata": {},
   "outputs": [],
   "source": [
    "# study = optuna.create_study(\n",
    "#     direction=\"maximize\"\n",
    "# )\n",
    "# study.optimize(optimise_catboost, timeout=60*60*6)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 33,
   "id": "40a89502-9df7-4ae8-9277-abd02f643fd2",
   "metadata": {},
   "outputs": [],
   "source": [
    "model_config_cb = ModelConfigCB(**{\n",
    "     'learning_rate': 0.02606161517843435,\n",
    "     'l2_leaf_reg': 18.04422276698195,\n",
    "     'random_strength': 4.7069580783889995,\n",
    "     'depth': 6,\n",
    "     'bagging_temperature': 0.8735940473548339,\n",
    "     'border_count': 256,\n",
    "     'grow_policy': 'Lossguide',\n",
    "     'min_data_in_leaf': 38})\n",
    "\n",
    "\n",
    "kaggle_model_config = {\n",
    "    'loss_function':     'Logloss',\n",
    "    'iterations':        200,\n",
    "    'verbose':           False,\n",
    "    'random_state':      seed,\n",
    "    'max_depth':         7, \n",
    "    'learning_rate':     0.06936242010150652, \n",
    "    'scale_pos_weight':  2.6149345838209532, \n",
    "    'l2_leaf_reg':       6.216113851699493, \n",
    "    'subsample':         0.6249261779711819, \n",
    "    'min_data_in_leaf':  24,\n",
    "    'cat_features':      cat_cols,\n",
    "}\n",
    "\n",
    "columns_to_drop=['tbp_lv_B',\n",
    " 'tbp_lv_C',\n",
    " 'tbp_lv_H',\n",
    " 'tbp_lv_L',\n",
    " 'tbp_lv_radial_color_std_max',\n",
    " 'tbp_lv_y',\n",
    " 'tbp_lv_z',\n",
    " 'luminance_contrast',\n",
    " 'lesion_color_difference',\n",
    " 'normalized_lesion_size',\n",
    " 'tbp_lv_norm_border_patient_norm',\n",
    " 'lesion_color_difference_patient_norm',\n",
    " 'age_normalized_nevi_confidence_2_patient_norm',\n",
    " 'tbp_lv_deltaA',\n",
    "\n",
    "'size_age_interaction', 'tbp_lv_deltaB', 'oof_eva_score__cluster',\n",
    "'hue_contrast_patient_norm__cluster',\n",
    "'color_contrast_index_patient_norm', 'tbp_lv_Lext__cluster',\n",
    "'tbp_lv_Cext_patient_norm', 'oof_eva_score_m',\n",
    "'age_approx_patient_norm', 'volume_approximation_3d_patient_norm'\n",
    "] "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 34,
   "id": "7d380ef0-b9f3-4ba3-a4f8-08fa19e57b6c",
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Default metric period is 5 because AUC is/are not implemented for GPU\n",
      "Default metric period is 5 because AUC is/are not implemented for GPU\n",
      "Default metric period is 5 because AUC is/are not implemented for GPU\n",
      "Default metric period is 5 because AUC is/are not implemented for GPU\n",
      "Default metric period is 5 because AUC is/are not implemented for GPU\n",
      "Default metric period is 5 because AUC is/are not implemented for GPU\n",
      "Default metric period is 5 because AUC is/are not implemented for GPU\n",
      "Default metric period is 5 because AUC is/are not implemented for GPU\n",
      "Default metric period is 5 because AUC is/are not implemented for GPU\n",
      "Default metric period is 5 because AUC is/are not implemented for GPU\n",
      "Default metric period is 5 because AUC is/are not implemented for GPU\n",
      "Default metric period is 5 because AUC is/are not implemented for GPU\n",
      "Default metric period is 5 because AUC is/are not implemented for GPU\n",
      "Default metric period is 5 because AUC is/are not implemented for GPU\n",
      "Default metric period is 5 because AUC is/are not implemented for GPU\n",
      "Default metric period is 5 because AUC is/are not implemented for GPU\n",
      "Default metric period is 5 because AUC is/are not implemented for GPU\n",
      "Default metric period is 5 because AUC is/are not implemented for GPU\n",
      "Default metric period is 5 because AUC is/are not implemented for GPU\n",
      "Default metric period is 5 because AUC is/are not implemented for GPU\n",
      "Default metric period is 5 because AUC is/are not implemented for GPU\n",
      "Default metric period is 5 because AUC is/are not implemented for GPU\n",
      "Default metric period is 5 because AUC is/are not implemented for GPU\n",
      "Default metric period is 5 because AUC is/are not implemented for GPU\n",
      "Default metric period is 5 because AUC is/are not implemented for GPU\n",
      "Default metric period is 5 because AUC is/are not implemented for GPU\n",
      "Default metric period is 5 because AUC is/are not implemented for GPU\n",
      "Default metric period is 5 because AUC is/are not implemented for GPU\n",
      "Default metric period is 5 because AUC is/are not implemented for GPU\n",
      "Default metric period is 5 because AUC is/are not implemented for GPU\n",
      "Default metric period is 5 because AUC is/are not implemented for GPU\n",
      "Default metric period is 5 because AUC is/are not implemented for GPU\n",
      "Default metric period is 5 because AUC is/are not implemented for GPU\n",
      "Default metric period is 5 because AUC is/are not implemented for GPU\n",
      "Default metric period is 5 because AUC is/are not implemented for GPU\n",
      "Default metric period is 5 because AUC is/are not implemented for GPU\n",
      "Default metric period is 5 because AUC is/are not implemented for GPU\n",
      "Default metric period is 5 because AUC is/are not implemented for GPU\n",
      "Default metric period is 5 because AUC is/are not implemented for GPU\n",
      "Default metric period is 5 because AUC is/are not implemented for GPU\n",
      "Default metric period is 5 because AUC is/are not implemented for GPU\n",
      "Default metric period is 5 because AUC is/are not implemented for GPU\n",
      "Default metric period is 5 because AUC is/are not implemented for GPU\n",
      "Default metric period is 5 because AUC is/are not implemented for GPU\n",
      "Default metric period is 5 because AUC is/are not implemented for GPU\n"
     ]
    }
   ],
   "source": [
    "scores_base, models, oof_values, fe_raw = optimise_catboost(model_config_cb.dict(), test=True, reduce=False,\n",
    "    columns_to_drop=columns_to_drop,\n",
    "    cat_cols=cat_cols, get_fe=False)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "5f182552-bf7c-40ca-a5bb-1698a675e188",
   "metadata": {},
   "outputs": [],
   "source": [
    "np.mean(scores_base)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "ca94883a-bd8e-43bf-a0fc-0e918545e190",
   "metadata": {},
   "outputs": [],
   "source": [
    "# fe_df = pd.DataFrame({\n",
    "#     \"feature_cols\": [i for i in feature_cols if i not in columns_to_drop],\n",
    "#     \"mean\": np.mean(np.concatenate([i['importances'] for i in fe_raw], axis=1), axis=1),\n",
    "#     \"std\": np.std(np.concatenate([i['importances'] for i in fe_raw], axis=1), axis=1)})\n",
    "# fe_df = fe_df.sort_values(by='mean', ascending=False).reset_index(drop=True)\n",
    "# fe_df.head(2)"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3 (ipykernel)",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.11.9"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 5
}
